﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.IO;
using System.Xml;
using System.Data.SqlClient;
using System.Configuration;

namespace WHTCWordCollector
{
    class WordTitle
    {
        public WordTitle(string word, string title, string link) { Word = word; Title = title; Link = link; }
        public string Word { get; set; }
        public string Title { get; set; }
        public string Link { get; set; }
    }


    class Program
    {
        static void Main(string[] args)
        {
            string[] urls = new string[] {
                "http://www.inspectiejeugdzorg.nl/abonneren/feeds/nieuwsberichten/", 
                "http://www.inspectiejeugdzorg.nl/abonneren/feeds/publicaties/",
                "http://www.werkinjeugdzorg.nl/applicaties/rss-feed.aspx",
                "http://feeds.rijksoverheid.nl/onderwerpen/jeugdzorg/nieuws.rss",
                "http://www.nji.nl/rssnieuws",
                "http://www.zorgenwelzijnplein.nl/index.php?page=resultaten&optie%5B3%5D%5B25%5D=on&optie_all%5B5%5D=on&optie_all%5B1%5D=on&submit_zoeken=Zoeken&p=1149&rss=true",
                "https://news.google.com/news/feeds?pz=1&cf=all&ned=nl_nl&hl=nl&q=jeugdzorg&output=rss"
            };

            Dictionary<string, int> words = new Dictionary<string, int>();
            List<WordTitle> titles = new List<WordTitle>();

            foreach (string url in urls)
            {
                try
                {
                    WebClient wc = new WebClient();
                    Stream s = wc.OpenRead(url);
                    XmlTextReader reader = new XmlTextReader(s);
                    while (reader.Read())
                    {
                        if (reader.IsStartElement("item"))
                        {
                            string title = "";
                            string link = "";
                            if (reader.ReadToDescendant("title"))
                                title = reader.ReadString();
                            if (reader.ReadToNextSibling("link"))
                                link = reader.ReadString();

                            Console.WriteLine(title);
                            Console.WriteLine(link);

                            string[] wordArray = title.Split(' ');
                            foreach (string w in wordArray)
                            {
                                char[] punctuation = new char[] { '.', '\'', '"', '#', ':', ',', '!', '?', ' ', '(', ')' };
                                string word = w.TrimStart(punctuation).TrimEnd(punctuation);
                                decimal d;
                                if (System.Decimal.TryParse(word, out d)) continue;
                                if (word.Length < 3) continue;

                                if (words.ContainsKey(word)) words[word]++;
                                else words.Add(word, 1);

                                titles.Add(new WordTitle(word, title, link));
                                Console.WriteLine("   {0}: {1}", word, words[word]);
                            }
                        }
                    }
                }
                catch (Exception ex)
                {
                    Console.WriteLine("Error: " + ex.Message);
                }
            }

            using (SqlConnection conn = new SqlConnection(ConfigurationManager.ConnectionStrings["WHTCDb"].ConnectionString))
            {
                conn.Open();
                SqlCommand cmdSaveWord = new SqlCommand("SaveWord", conn);
                SqlCommand cmdSaveTitle = new SqlCommand("SaveTitle", conn);

                cmdSaveWord.CommandType = System.Data.CommandType.StoredProcedure;
                cmdSaveTitle.CommandType = System.Data.CommandType.StoredProcedure;
                
                foreach (string w in words.Keys)
                {
                    cmdSaveWord.Parameters.Clear();
                    cmdSaveWord.Parameters.AddWithValue("@word", w);
                    cmdSaveWord.Parameters.AddWithValue("@hits", words[w]);
                    cmdSaveWord.ExecuteNonQuery();

                    Console.WriteLine("inserting " + w);
                }

                foreach (WordTitle t in titles)
                {
                    cmdSaveTitle.Parameters.Clear();
                    cmdSaveTitle.Parameters.AddWithValue("@word", t.Word);
                    cmdSaveTitle.Parameters.AddWithValue("@title", t.Title);
                    cmdSaveTitle.Parameters.AddWithValue("@link", t.Link);

                    cmdSaveTitle.ExecuteNonQuery();

                    Console.WriteLine("Inserting " + t.Title);
                }
            }
        }
    }
}
